package com.study.spark.ml.sample

import org.apache.spark.mllib.feature.{HashingTF, IDF}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * 计算文件中每个词的TF-IDF值
  * 体现一个单词对于一个文档集或语料库中的某一个文档的重要程度
  * @author: stephen.shen
  * @create: 2019-04-12 17:33
  */
object TfIdfSample {

  def main(args: Array[String]): Unit = {
    val file = "D:\\ProgramFiles\\spark-2.3.1\\README.md"
    val conf = new SparkConf().setAppName("TF-IDF Sample").setMaster("local")
    val sc = new SparkContext(conf)
    val documents = sc.textFile(file).map(_.split(" ").toSeq)
    println("Documents Size:" + documents.count())

    val hashingTF = new HashingTF()
    val tf = hashingTF.transform(documents)
    for (_tf <- tf) {
      println(_tf)
    }

    tf.cache()
    val idf = new IDF().fit(tf)
    val tfidf = idf.transform(tf)
    println("tfidf size: " + tfidf.count())

    for (_tfidf <- tfidf) {
      println(_tfidf)
    }
  }
}
